from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import gc
import math
import os
import random
import sys
import time

def myTest2():
	"""docstring for myTest2"""
	str1 == "apple"

def myTest():
	"""This is a test function from home"""
	pass
# function listings:
# differencesOfResultsBetweenANDAndORSemanticsOf95KQueries()
# differencesOfResultsBetweenANDAndORSemanticsOfHJQueries()
# getResultsAndComputeOverlapAgainstUnprunedIndex()
# getResultsAndComputePAt10AtDifferentPruningLevels()
# generateRandomDocids()
# compute_verify_overlap_for_Xdoc_method()
# TODO

def differencesOfResultsBetweenANDAndORSemanticsOf95KQueries():
    docIDsDictFromFile1 = {}
    docIDsDictFromFile2 = {}
    
    # AND case
    inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/rawResults_uniform_impactScore_listLength_100%_TOP10_AND_20131210Night"
    # OR case
    inputFileName2 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/rawResults_50%_TOP10_OR_20140126Night"
    print "inputFileName1:",inputFileName1
    print "inputFileName2:",inputFileName2
    
    inputFileHanlder1 = open(inputFileName1,"r")
    inputFileHanlder2 = open(inputFileName2,"r")

    currentQueryID = ""
    currentLine = inputFileHanlder1.readline()
    numOfTOP10DocumentResults1 = 0
    currentRank = 0
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        if currentLine.strip().startswith("qid:"):
            print currentLineElements
            currentQueryID = currentLineElements[1]
            currentRank = 0
        
        if len(currentLineElements) == 14 and currentLineElements[-1].startswith("GX"):
            currentRank += 1
            if currentRank <= 10:
                # print currentRank,currentLine.strip()
                currentDocID = currentLineElements[-2]
                if currentDocID not in docIDsDictFromFile1:
                    docIDsDictFromFile1[currentDocID] = 1
                else:
                    docIDsDictFromFile1[currentDocID] += 1
                numOfTOP10DocumentResults1 += 1
        currentLine = inputFileHanlder1.readline()
    
    
    currentQueryID = ""
    currentLine = inputFileHanlder2.readline()
    numOfTOP10DocumentResults2 = 0
    currentRank = 0
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        if currentLine.strip().startswith("qid:"):
            print currentLineElements
            currentQueryID = currentLineElements[1]
            currentRank = 0
        
        if len(currentLineElements) == 25 and currentLineElements[-2].startswith("GX"):
            currentRank += 1
            if currentRank <= 10:
                # print currentRank,currentLine.strip()
                currentDocID = currentLineElements[-3]
                if currentDocID not in docIDsDictFromFile2:
                    docIDsDictFromFile2[currentDocID] = 1
                else:
                    docIDsDictFromFile2[currentDocID] += 1
                numOfTOP10DocumentResults2 += 1
        currentLine = inputFileHanlder2.readline()
    
    print "numOfTOP10DocumentResultsFromANDHead95K:",numOfTOP10DocumentResults1
    print "numOfTOP10DocumentResultsFromORHead95K:",numOfTOP10DocumentResults2
    intersectionSet = set(docIDsDictFromFile1).intersection( set(docIDsDictFromFile2) )
    unionSet = set(docIDsDictFromFile1).union( set(docIDsDictFromFile2) )
    print "len(intersectionSet):",len(intersectionSet)
    print "len(unionSet):",len(unionSet)
    print "symmetric difference:",len(intersectionSet)/len(unionSet)

def differencesOfResultsBetweenANDAndORSemanticsOfHJQueries():
    trecIDsDictFromFile1 = {}
    trecIDsDictFromFile2 = {}
    # AND
    inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/AND_top100_100%_tb04-06_final_with_termID_and_score_added"
    # OR
    inputFileName2 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/OR_top100_100%_tb04-06_final_with_termID_and_score_added"
    inputFileHanlder1 = open(inputFileName1,"r")
    inputFileHanlder2 = open(inputFileName2,"r")
    
    for line in inputFileHanlder1.readlines():
        lineElements = line.strip().split(" ")
        currentTrecID = lineElements[2]
        currentRank = int(lineElements[3])
        if currentRank <= 10:
            if currentTrecID not in trecIDsDictFromFile1:
                trecIDsDictFromFile1[currentTrecID] = 1
            else:
                trecIDsDictFromFile1[currentTrecID] += 1

    for line in inputFileHanlder2.readlines():
        lineElements = line.strip().split(" ")
        currentTrecID = lineElements[2]
        currentRank = int(lineElements[3])
        if currentRank <= 10:
            if currentTrecID not in trecIDsDictFromFile2:
                trecIDsDictFromFile2[currentTrecID] = 1
            else:
                trecIDsDictFromFile2[currentTrecID] += 1

    print "len(trecIDsDictFromFile1):",len(trecIDsDictFromFile1)
    print "len(trecIDsDictFromFile2):",len(trecIDsDictFromFile2)

    intersectionSet = set(trecIDsDictFromFile1).intersection( set(trecIDsDictFromFile2) )
    unionSet = set(trecIDsDictFromFile1).union( set(trecIDsDictFromFile2) )
    print "len(intersectionSet):",len(intersectionSet)
    print "len(unionSet):",len(unionSet)
    inputFileHanlder1.close()
    inputFileHanlder2.close()

def getResultsAndComputeOverlapAgainstUnprunedIndex():
    numOfResultsReturnedByTheCurrentTieringMethod = 0
    
    numOfDocumentsInTotalInGov2 = 25205179
    numOfDocsAtDebugPercent = 10
    numOfDocsAt1Percent = int(numOfDocumentsInTotalInGov2 * 0.01)
    numOfDocsAt3Percent = int(numOfDocumentsInTotalInGov2 * 0.03)
    numOfDocsAt5Percent = int(numOfDocumentsInTotalInGov2 * 0.05)
    numOfDocsAt10Percent = int(numOfDocumentsInTotalInGov2 * 0.1)
    numOfDocsAt15Percent = int(numOfDocumentsInTotalInGov2 * 0.15)
    numOfDocsAt20Percent = int(numOfDocumentsInTotalInGov2 * 0.2)
    numOfDocsAt30Percent = int(numOfDocumentsInTotalInGov2 * 0.3)
    numOfDocsAt40Percent = int(numOfDocumentsInTotalInGov2 * 0.4)
    numOfDocsAt50Percent = int(numOfDocumentsInTotalInGov2 * 0.5)
    print "numOfDocsAtDebugPercent:",numOfDocsAtDebugPercent
    print "numOfDocsAt1Percent:",numOfDocsAt1Percent
    print "numOfDocsAt3Percent:",numOfDocsAt3Percent
    print "numOfDocsAt5Percent:",numOfDocsAt5Percent
    print "numOfDocsAt10Percent:",numOfDocsAt10Percent
    print "numOfDocsAt15Percent:",numOfDocsAt15Percent
    print "numOfDocsAt20Percent:",numOfDocsAt20Percent
    print "numOfDocsAt30Percent:",numOfDocsAt30Percent
    print "numOfDocsAt40Percent:",numOfDocsAt40Percent
    print "numOfDocsAt50Percent:",numOfDocsAt50Percent
    
    # option1:
    # inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/randomDocumentsSelectedAtDifferentPercentage_20140412"
    # option2:
    inputFileName1 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/gov2_Docs_with_TheirXdocValues_Since20140428_sortedByXdocValues"
    inputFileHanlder = open(inputFileName1,"r")
    currentLine = inputFileHanlder.readline()

    docIDDict = {}
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        currentDocID = currentLineElements[1]
        if currentDocID not in docIDDict:
            docIDDict[currentDocID] = 1
        else:
            print "duplicated docID detected."
            exit(1)
        if len(docIDDict) == numOfDocsAt50Percent:
            break
        currentLine = inputFileHanlder.readline()
    print "len(docIDDict):",len(docIDDict)
    inputFileHanlder.close()
    
    
    numOfResultsReturned = 0
    # key: qID
    # value: list of document candidates
    qIDWithTOP10DocumentResultDict = {}
    # dodo
    inputFileName2 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/rawResultFile_150_human_queries_top2M_AND_20140412"
    # Jiahui's computer
    # inputFileName2 = "/Users/wc3045963/Desktop/programs_20140412/rawResultFile_150_human_queries_top2M_AND_20140412"
    inputFileHanlder2 = open(inputFileName2,"r")
    currentQueryID = ""
    currentLine = inputFileHanlder2.readline()
    while currentLine:
        if currentLine.strip().startswith("qid:"):
            currentLineElements = currentLine.strip().split(" ")
            print currentLineElements
            currentQueryID = currentLineElements[1]
            
            # for debug
            # if currentQueryID == "704":
            #    break
            
            if currentQueryID not in qIDWithTOP10DocumentResultDict:
                qIDWithTOP10DocumentResultDict[currentQueryID] = []

        if currentLine.strip().startswith("Score:"):
            currentLineElements = currentLine.strip().split("\t")
            currentDocIDStringPair = currentLineElements[1]
            currentTrecIDStringPair = currentLineElements[2]
            currentDocID = currentDocIDStringPair.strip().split(":")[1].strip()
            currentTrecID = currentTrecIDStringPair.strip().split(":")[1].strip()
            # print currentDocID,currentTrecID

            if len(qIDWithTOP10DocumentResultDict[currentQueryID]) < 10:
                qIDWithTOP10DocumentResultDict[currentQueryID].append(currentDocID)
                if currentDocID not in docIDDict:
                    pass
                else:
                    numOfResultsReturnedByTheCurrentTieringMethod += 1
                numOfResultsReturned += 1
            else:
                pass

        currentLine = inputFileHanlder2.readline()
    
    print "numOfResultsReturned:",numOfResultsReturned
    print "numOfResultsReturnedByTheCurrentTieringMethod:",numOfResultsReturnedByTheCurrentTieringMethod
    print "len(qIDWithTOP10DocumentResultDict):",len(qIDWithTOP10DocumentResultDict)
    print "qIDWithTOP10DocumentResultDict['701']:",qIDWithTOP10DocumentResultDict['701']
    inputFileHanlder2.close()
    
def getResultsAndComputePAt10AtDifferentPruningLevels(numOfDocumentsIncluded):
    # Please CHANGE the following places:
    # variable: numOfDocumentsInCollection
    # inputFileName0
    # inputFileName1
    # inputFileName2
    
    numOfRelevantDocumentResultsCaptured = 0
    numOfRelevantDocumentResultsCapturedInTOP10 = 0
    numOfDocumentResultsCapturedInOriginalTOP10 = 0
    
    # measure in % of the index in terms of # of postings in total
    # for gov2
    numOfDocumentsInCollection = 25205179
    # for clueweb09B
    # numOfDocumentsInCollection = 50220423
    
    # key: trec id
    # value: the # of times this document has been used
    relevant_doc_result_dict = {}
    qid_dict = {}
    relevant_label_dict = {}
    # for gov2
    # on dodo
    # inputFileName0 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/qrels.gov2.all"
    # on vidaserver1
    inputFileName0 = "/local_scratch/wei/workspace/NYU_IRTK/results/dynamicUnigramFromWei/gov2/qrels.gov2.all"
    
    # for clueweb09B
    # year 2009
    # inputFileName0 = "/local_scratch/wei/workspace/NYU_IRTK/results/dynamicUnigramFromWei/clueweb09B/qrels.web09catA.txt"
    # year 2009 - 2012
    # inputFileName0 = "/local_scratch/wei/workspace/NYU_IRTK/results/dynamicUnigramFromWei/clueweb09B/qrels.web09_12catA.txt"
    input_file_handler_3 = open(inputFileName0,"r")
    for line in input_file_handler_3.readlines():
        line_elements = line.strip().split(" ")
        q_id = line_elements[0]
        trec_id = line_elements[2]
        currentDocumentResultKey = q_id + "_" + trec_id
        relevance_label = line_elements[3]
        # print line.strip()
        if q_id not in qid_dict:
            qid_dict[q_id] = 1
        if relevance_label not in relevant_label_dict:
            relevant_label_dict[relevance_label] = 1
        if int(relevance_label) > 0:
            if currentDocumentResultKey not in relevant_doc_result_dict:
                relevant_doc_result_dict[currentDocumentResultKey] = 1
            else:
                relevant_doc_result_dict[currentDocumentResultKey] += 1
    input_file_handler_3.close()
    print "len(relevant_doc_result_dict):",len(relevant_doc_result_dict)
    print "qid_dict:",len(qid_dict)
    print "list(relevant_label_dict):",list(relevant_label_dict)

    
    # 200 0 clueweb09-enwp02-24-19721 2
    # print "relevant_doc_result_dict['200_clueweb09-enwp02-24-19721']:",relevant_doc_result_dict['200_clueweb09-enwp02-24-19721']

    # for gov2
    # option1:
    # inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/randomDocumentsSelectedAtDifferentPercentage_20140412"
    # option2:
    # inputFileName1 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/gov2_Docs_with_TheirXdocValues_Since20140428_sortedByXdocValues"
    # inputFileName1 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/gov2_Docs_with_TheirXdocValues_Since20140428_sortedByXdocValues"
    # option3:
    # inputFileName1 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/trecID_docID_numOfPostingsRecorded_DocSizeInWords_MappingTableForGov2Dataset_with_Xdoc_values_added_sortedByXdocValueUsingGoodTurningDividedByNumOfPostingsForEachDocument"
    # option4:
    # inputFileName1 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/trecID_docID_numOfPostingsRecorded_DocSizeInWords_Xdocs_and_variations_sortedByXdocDividedBySqrtNum_20150506"
    # option5:
    # inputFileName1 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/trecID_docID_numOfPostingsRecorded_DocSizeInWords_Xdocs_and_variations_sortedByXdocDividedByLogNum_20150505"
    # option6:
    # inputFileName1 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/trecID_docID_numOfPostingsRecorded_DocSizeInWords_Xdocs_and_variations_sortedByXdocMultipleByLogNum_20150505"
    # option7:
    # sorted by # of postings / unique terms
    # inputFileName1 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/trecID_docID_numOfPostingsRecorded_DocSizeInWords_Xdocs_and_variations_sortedByNumOfPostings_20150519"
    # option8:
    # sorted by static probability SUM for each document (Updated by Wei on 2014/06/17)
    # inputFileName1 = "/home/vgc/wei/workspace/NYU_IRTK/data/Gov2_XdocCombinedWithPTOPK_WHOLE_20140617_sortedBySumStatic" 
    # Updated by Wei 2014/06/22
    # xDoc
    # inputFileName1 = "/home/vgc/wei/workspace/NYU_IRTK/data/Gov2_XdocCombinedWithPTOPK_WHOLE_20140620_sortedBySum_PTopK_PowTo0"
    # power to 1
    # inputFileName1 = "/home/vgc/wei/workspace/NYU_IRTK/data/Gov2_XdocCombinedWithPTOPK_WHOLE_20140620_sortedBySum_PTopK_PowTo1"
    
    # WRONG inputFileName1 = "/home/vgc/wei/workspace/NYU_IRTK/data/Gov2_XdocCombinedWithPTOPK_WHOLE_20140620_sortedBySum_PTopK_PowTo2"
    # WRONG inputFileName1 = "/home/vgc/wei/workspace/NYU_IRTK/data/Gov2_XdocCombinedWithPTOPK_WHOLE_20140620_sortedBySum_PTopK_PowTo3"
    # WRONG inputFileName1 = "/home/vgc/wei/workspace/NYU_IRTK/data/Gov2_XdocCombinedWithPTOPK_WHOLE_20140620_sortedBySum_PTopK_PowTo4"
    
    # Sum impact scores
    # inputFileName1 = "/home/vgc/wei/workspace/NYU_IRTK/data/Gov2_XdocCombinedWithPTOPK_WHOLE_20140620_sortedBySumImpactScores"
    # power to dot3
    # inputFileName1 = "/home/vgc/wei/workspace/NYU_IRTK/data/Gov2_XdocCombinedWithPTOPK_WHOLE_20140621_sortedBySum_PTopK_PowToDot3"
    # power to dot5
    # inputFileName1 = "/home/vgc/wei/workspace/NYU_IRTK/data/Gov2_XdocCombinedWithPTOPK_WHOLE_20140621_sortedBySum_PTopK_PowToDot5"
    # power to dot7
    inputFileName1 = "/home/vgc/wei/workspace/NYU_IRTK/data/Gov2_XdocCombinedWithPTOPK_WHOLE_20140621_sortedBySum_PTopK_PowToDot7"
    
    
    # for clueweb09B
    # option1:
    # sorted by xDoc
    # inputFileName1 = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/xDocValues_clueweb09B_20140611_sortedByXdoc"
    # sorted by xDoc * Log(# of postings)
    # inputFileName1 = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/xDocValues_variations_clueweb09B_20140611_sortedByXdocTimesLogNumOfPostings"
    # sorted by xDoc / Log(# of postings)
    # inputFileName1 = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/xDocValues_variations_clueweb09B_20140611_sortedByXdocDividedByLogNumOfPostings"
    # sorted by xDoc * SQRT(# of postings)
    # inputFileName1 = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/xDocValues_variations_clueweb09B_20140611_sortedByXdocTimesSQRTNumOfPostings"
    # sorted by xDoc / SQRT(# of postings)
    # inputFileName1 = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/xDocValues_variations_clueweb09B_20140611_sortedByXdocDividedBySQRTNumOfPostings"
    # sorted by # of postings
    # inputFileName1 = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/xDocValues_clueweb09B_20140611_sortedByNumOfPostings"
    
    inputFileHanlder = open(inputFileName1,"r")
    currentLine = inputFileHanlder.readline()

    docIDDict = {}
    trecIDDict = {}
    
    # for gov2
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        # Note: change for different format
        currentDocID = currentLineElements[0]
        
        if currentDocID not in docIDDict:
            docIDDict[currentDocID] = 1
        else:
            print "duplicated docID detected."
            exit(1)
        
        if len(docIDDict) == numOfDocumentsIncluded:
            break
        currentLine = inputFileHanlder.readline()
    print "len(docIDDict):",len(docIDDict)
    print "len(trecIDDict):",len(trecIDDict)
    inputFileHanlder.close()

    # key: qID
    # value: list of document candidates
    qIDWithTOP10DocumentResultDict = {}
    qIDWithTOP10DocumentOriginalRankDict = {}
    qIDWithRelevantDocumentResultDict = {}
    qIDWithRelevantDocumentOriginalRankDict = {}
    top10_document_result_dict = {}
    # value = 0 : file format 0
    # value = 1 : file format 1
    # value = 2 : file format 2
    
    # for gov2
    # AND semantics
    # inputFileName2 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/rawResultFile_150_human_queries_top2M_AND_20140412"
    # OR semantics
    # on dodo:
    # inputFileName2 = "/home/diaosi/web-search-engine-wei_MOVE_FROM_PANGOLIN_20131206/polyIRIndexer/rawResultFile_150_human_queries_top2M_OR_20140412"
    # on vidaserver1
    # AND semantics
    # inputFileName2 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/rawResultFile_150_human_queries_top2M_AND_20140412"
    # OR semantics    
    # TOP2M
    inputFileName2 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/rawResultFile_150_human_queries_top2M_OR_20140412"
    
    # for clueweb09B
    # AND semantics
    # year2009
    # TYPICAL WRONG EXAMPLE: DON'T USE ONLY TOP100 results as reference
    # WRONG inputFileName2 = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/clueweb09B_09WebTrack_rawResults_20140611_TOP100_AND"
    # inputFileName2 = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/clueweb09B_09WebTrack_rawResults_20140611_TOP2M_AND"
    # year09-12
    # WRONG inputFileName2 = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/clueweb09B_09_12WebTrack_rawResults_20140611_TOP100_AND"
    # inputFileName2 = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/clueweb09B_09_12_rawResults_20140611_TOP2M_AND"
    
    # OR semantics
    # year2009
    # TYPICAL WRONG EXAMPLE: DON'T USE ONLY TOP100 results as reference
    # WRONG inputFileName2 = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/clueweb09B_09WebTrack_rawResults_20140611_TOP100_OR"
    # inputFileName2 = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/clueweb09B_09WebTrack_rawResults_20140611_TOP2M_OR"
    # year09-12
    # WRONG inputFileName2 = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/clueweb09B_09_12WebTrack_rawResults_20140611_TOP100_OR"
    # inputFileName2 = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/clueweb09B_09_12_rawResults_20140611_TOP2M_OR"
    
    inputFileHanlder2 = open(inputFileName2,"r")
    currentQueryID = ""
    currentLine = inputFileHanlder2.readline()
    while currentLine:
        if currentLine.strip().startswith("qid:"):
            currentLineElements = currentLine.strip().split(" ")
            print "processing qid",currentLineElements[1]
            currentQueryID = currentLineElements[1]
            
            if currentQueryID not in qIDWithTOP10DocumentResultDict:
                qIDWithTOP10DocumentResultDict[currentQueryID] = []
                qIDWithTOP10DocumentOriginalRankDict[currentQueryID] = []
                qIDWithRelevantDocumentResultDict[currentQueryID] = []
                qIDWithRelevantDocumentOriginalRankDict[currentQueryID] = []
            
            currentRank = 0
            
        if currentLine.strip().startswith("Score:"):
            currentRank += 1
            currentLineElements = currentLine.strip().split("\t")
            currentScoreStringPair = currentLineElements[0]
            currentDocIDStringPair = currentLineElements[1]
            currentTrecIDStringPair = currentLineElements[2]
            currentScore = currentScoreStringPair.strip().split(":")[1].strip()
            currentDocID = currentDocIDStringPair.strip().split(":")[1].strip()
            currentTrecID = currentTrecIDStringPair.strip().split(":")[1].strip()
            # print currentDocID,currentTrecID
            
            # set a threshold to the documents we tend to measure
            if currentRank <= 2000000:
            # if currentRank <= 100:
                # compare method2:
                if currentDocID not in docIDDict:
                    pass
                else:
                    if len(qIDWithTOP10DocumentResultDict[currentQueryID]) < 10:
                        currentDocumentResultKey = currentQueryID + "_" + currentTrecID
                        qIDWithTOP10DocumentResultDict[currentQueryID].append(currentDocumentResultKey)
                        qIDWithTOP10DocumentOriginalRankDict[currentQueryID].append(currentRank)

                        outputLineForEval = str(currentQueryID) + " " + "Q0" + " " + str(currentTrecID) + " " + str( len(qIDWithTOP10DocumentOriginalRankDict[currentQueryID]) ) + " " + str(currentScore) + " " + "PolyIRTKDebug" + " " + str(currentRank) + "\n"
                        print outputLineForEval,
                        
                        if currentRank <= 10:
                            numOfDocumentResultsCapturedInOriginalTOP10 += 1
                        if currentDocumentResultKey in relevant_doc_result_dict:
                            numOfRelevantDocumentResultsCaptured += 1
                            qIDWithRelevantDocumentResultDict[currentQueryID].append(currentDocumentResultKey)
                            qIDWithRelevantDocumentOriginalRankDict[currentQueryID].append(currentRank)
                            if currentRank <= 10:
                                numOfRelevantDocumentResultsCapturedInTOP10 += 1
                    else:
                        pass
            
        currentLine = inputFileHanlder2.readline()
    
    print "len(qIDWithTOP10DocumentResultDict):",len(qIDWithTOP10DocumentResultDict)
    QIDList = []
    QIDList = qIDWithTOP10DocumentResultDict.keys()
    QIDList.sort(cmp=None, key=None, reverse=False)
    tempCounter = 0
    for currentQID in QIDList:
        print "********************"
        print currentQID
        print qIDWithTOP10DocumentResultDict[currentQID]
        print qIDWithTOP10DocumentOriginalRankDict[currentQID]
        print qIDWithRelevantDocumentResultDict[currentQID]
        print qIDWithRelevantDocumentOriginalRankDict[currentQID]
        tempCounter += len(qIDWithTOP10DocumentOriginalRankDict[currentQID])
    
    # print "qIDWithTOP10DocumentResultDict['801']:",qIDWithTOP10DocumentResultDict['801']
    # print "qIDWithTOP10DocumentResultDict['804']:",qIDWithTOP10DocumentResultDict['804']
    print "len(top10_document_result_dict):",len(top10_document_result_dict)
    
    inputFileHanlder2.close()

    # do the intersection and see the %
    print "len( set(relevant_doc_result_dict) ):",len( set(relevant_doc_result_dict) )
    print "numOfRelevantDocumentResultsCaptured numOfRelevantDocumentResultsCapturedInTOP10 numOfDocumentResultsCapturedInOriginalTOP10 numOfDocumentResultsReturned"
    print numOfRelevantDocumentResultsCaptured,numOfRelevantDocumentResultsCapturedInTOP10,numOfDocumentResultsCapturedInOriginalTOP10,tempCounter
    print "Overall:"
    print "# Of Documents In Collection:",numOfDocumentsInCollection
    print "inputFileName0:",inputFileName0
    print "inputFileName1:",inputFileName1
    print "inputFileName2:",inputFileName2

def getResultsAndComputePAt10ForDocHits(inputFileName1):
    print "inputFileName1:",inputFileName1
    numOfRelevantDocumentResultsCaptured = 0
    numOfRelevantDocumentResultsCapturedInTOP10 = 0
    numOfDocumentResultsCapturedInOriginalTOP10 = 0
    
    # measure in % of the index in terms of # of postings in total
    # for gov2
    numOfDocumentsInCollection = 25205179
    totalNumOfPostings = 6451948010
    fivePercentOfTotalNumOfPostings = totalNumOfPostings * 0.05
	
    # key: trec id
    # value: the # of times this document has been used
    relevant_doc_result_dict = {}
    qid_dict = {}
    relevant_label_dict = {}
    # for gov2
    # on vidaserver1
    inputFileName0 = "/home/vgc/wei/workspace/NYU_IRTK/data/qrels.gov2.all"
    input_file_handler_3 = open(inputFileName0,"r")
    for line in input_file_handler_3.readlines():
        line_elements = line.strip().split(" ")
        q_id = line_elements[0]
        trec_id = line_elements[2]
        currentDocumentResultKey = q_id + "_" + trec_id
        relevance_label = line_elements[3]
        # print line.strip()
        if q_id not in qid_dict:
            qid_dict[q_id] = 1
        if relevance_label not in relevant_label_dict:
            relevant_label_dict[relevance_label] = 1
        if int(relevance_label) > 0:
            if currentDocumentResultKey not in relevant_doc_result_dict:
                relevant_doc_result_dict[currentDocumentResultKey] = 1
            else:
                relevant_doc_result_dict[currentDocumentResultKey] += 1
    input_file_handler_3.close()
    print "len(relevant_doc_result_dict):",len(relevant_doc_result_dict)
    print "qid_dict:",len(qid_dict)
    print "list(relevant_label_dict):",list(relevant_label_dict)

    # for gov2
    docIDDict = {}
    inputFileHanlder = open(inputFileName1,"r")
    currentLine = inputFileHanlder.readline()
    # for gov2
    sumOfPostings = 0
    while currentLine:
        le = currentLine.strip().split(" ")
        docID = le[0]
        # print "docID:",docID
        sumOfPostings += int(le[2])
        docIDDict[docID] = 1
        if sumOfPostings > fivePercentOfTotalNumOfPostings:
        	break
        currentLine = inputFileHanlder.readline()
    print "len(docIDDict):",len(docIDDict),sumOfPostings
    inputFileHanlder.close()
	
    # key: qID
    # value: list of document candidates
    qIDWithTOP10DocumentResultDict = {}
    qIDWithTOP10DocumentOriginalRankDict = {}
    qIDWithRelevantDocumentResultDict = {}
    qIDWithRelevantDocumentOriginalRankDict = {}
    top10_document_result_dict = {}
    
    # for gov2
    # OR semantics    
    # TOP2M
    inputFileName2 = "/home/vgc/wei/workspace/NYU_IRTK/results/rawResultFile_150_human_queries_top2M_OR_Gov2_20140412"
    inputFileHanlder2 = open(inputFileName2,"r")
    currentQueryID = ""
    currentLine = inputFileHanlder2.readline()
    while currentLine:
        if currentLine.strip().startswith("qid:"):
            currentLineElements = currentLine.strip().split(" ")
            print "processing qid",currentLineElements[1]
            currentQueryID = currentLineElements[1]
            
            if currentQueryID not in qIDWithTOP10DocumentResultDict:
                qIDWithTOP10DocumentResultDict[currentQueryID] = []
                qIDWithTOP10DocumentOriginalRankDict[currentQueryID] = []
                qIDWithRelevantDocumentResultDict[currentQueryID] = []
                qIDWithRelevantDocumentOriginalRankDict[currentQueryID] = []
            
            currentRank = 0
            
        if currentLine.strip().startswith("Score:"):
            currentRank += 1
            currentLineElements = currentLine.strip().split("\t")
            currentScoreStringPair = currentLineElements[0]
            currentDocIDStringPair = currentLineElements[1]
            currentTrecIDStringPair = currentLineElements[2]
            currentScore = currentScoreStringPair.strip().split(":")[1].strip()
            currentDocID = currentDocIDStringPair.strip().split(":")[1].strip()
            currentTrecID = currentTrecIDStringPair.strip().split(":")[1].strip()
            # print currentDocID,currentTrecID
            
            # set a threshold to the documents we tend to measure
            if currentRank <= 2000000:
            # if currentRank <= 100:
                # compare method2:
                if currentDocID not in docIDDict:
                    pass
                else:
                    if len(qIDWithTOP10DocumentResultDict[currentQueryID]) < 10:
                        currentDocumentResultKey = currentQueryID + "_" + currentTrecID
                        qIDWithTOP10DocumentResultDict[currentQueryID].append(currentDocumentResultKey)
                        qIDWithTOP10DocumentOriginalRankDict[currentQueryID].append(currentRank)

                        outputLineForEval = str(currentQueryID) + " " + "Q0" + " " + str(currentTrecID) + " " + str( len(qIDWithTOP10DocumentOriginalRankDict[currentQueryID]) ) + " " + str(currentScore) + " " + "NYUIRTK" + " " + str(currentRank) + " " + str(currentDocID) + "\n"
                        print outputLineForEval,
                        
                        if currentRank <= 10:
                            numOfDocumentResultsCapturedInOriginalTOP10 += 1
                        if currentDocumentResultKey in relevant_doc_result_dict:
                            numOfRelevantDocumentResultsCaptured += 1
                            qIDWithRelevantDocumentResultDict[currentQueryID].append(currentDocumentResultKey)
                            qIDWithRelevantDocumentOriginalRankDict[currentQueryID].append(currentRank)
                            if currentRank <= 10:
                                numOfRelevantDocumentResultsCapturedInTOP10 += 1
                    else:
                        pass
            
        currentLine = inputFileHanlder2.readline()
    
    print "len(qIDWithTOP10DocumentResultDict):",len(qIDWithTOP10DocumentResultDict)
    QIDList = []
    QIDList = qIDWithTOP10DocumentResultDict.keys()
    QIDList.sort(cmp=None, key=None, reverse=False)
    tempCounter = 0
    for currentQID in QIDList:
        print "********************"
        print currentQID
        print qIDWithTOP10DocumentResultDict[currentQID]
        print qIDWithTOP10DocumentOriginalRankDict[currentQID]
        print qIDWithRelevantDocumentResultDict[currentQID]
        print qIDWithRelevantDocumentOriginalRankDict[currentQID]
        tempCounter += len(qIDWithTOP10DocumentOriginalRankDict[currentQID])
    
    # print "qIDWithTOP10DocumentResultDict['801']:",qIDWithTOP10DocumentResultDict['801']
    # print "qIDWithTOP10DocumentResultDict['804']:",qIDWithTOP10DocumentResultDict['804']
    print "len(top10_document_result_dict):",len(top10_document_result_dict)
    
    inputFileHanlder2.close()

    # do the intersection and see the %
    print "len( set(relevant_doc_result_dict) ):",len( set(relevant_doc_result_dict) )
    print "numOfRelevantDocumentResultsCaptured numOfRelevantDocumentResultsCapturedInTOP10 numOfDocumentResultsCapturedInOriginalTOP10 numOfDocumentResultsReturned"
    print numOfRelevantDocumentResultsCaptured,numOfRelevantDocumentResultsCapturedInTOP10,numOfDocumentResultsCapturedInOriginalTOP10,tempCounter
    print "Overall:"
    print "# Of Documents In Collection:",numOfDocumentsInCollection
    print "inputFileName0:",inputFileName0
    print "inputFileName1:",inputFileName1
    print "inputFileName2:",inputFileName2
    
# for the use of random document partition for tiering
def generateRandomDocids():
    print "Begins..."
    # for moa:
    outputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-April/data/randomDocumentsSelectedAtDifferentPercentage_20140412"
    outputFileHandler = open(outputFileName,"w")
    numOfDocumentsInTotalInGov2 = 25205179
    docCollectionLowerBound = 0
    docCollectionUpperBound = 25205178
    numOfDocsAtDebugPercent = 10
    numOfDocsAt1Percent = int(numOfDocumentsInTotalInGov2 * 0.01)
    numOfDocsAt3Percent = int(numOfDocumentsInTotalInGov2 * 0.03)
    numOfDocsAt5Percent = int(numOfDocumentsInTotalInGov2 * 0.05)
    numOfDocsAt10Percent = int(numOfDocumentsInTotalInGov2 * 0.1)
    numOfDocsAt15Percent = int(numOfDocumentsInTotalInGov2 * 0.15)
    numOfDocsAt20Percent = int(numOfDocumentsInTotalInGov2 * 0.2)
    numOfDocsAt30Percent = int(numOfDocumentsInTotalInGov2 * 0.3)
    numOfDocsAt40Percent = int(numOfDocumentsInTotalInGov2 * 0.4)
    numOfDocsAt50Percent = int(numOfDocumentsInTotalInGov2 * 0.5)
    print "numOfDocsAtDebugPercent:",numOfDocsAtDebugPercent
    print "numOfDocsAt1Percent:",numOfDocsAt1Percent
    print "numOfDocsAt3Percent:",numOfDocsAt3Percent
    print "numOfDocsAt5Percent:",numOfDocsAt5Percent
    print "numOfDocsAt10Percent:",numOfDocsAt10Percent
    print "numOfDocsAt15Percent:",numOfDocsAt15Percent
    print "numOfDocsAt20Percent:",numOfDocsAt20Percent
    print "numOfDocsAt30Percent:",numOfDocsAt30Percent
    print "numOfDocsAt40Percent:",numOfDocsAt40Percent
    print "numOfDocsAt50Percent:",numOfDocsAt50Percent

    docIDDict = {}
    docIDList = []

    while True:
        currentDocIDSelected = random.randint(docCollectionLowerBound, docCollectionUpperBound)
        if currentDocIDSelected not in docIDDict:
            docIDDict[currentDocIDSelected] = 1
            docIDList.append(currentDocIDSelected)
        if len(docIDDict) == numOfDocsAtDebugPercent:
            break

    for index,docID in enumerate(docIDList):
        outputFileHandler.write(str(index) + " " + str(docID) + "\n")
    print "Overall:"
    print "outputFileName:",outputFileName
    outputFileHandler.close()
    print "Ends."

print "Begins..."
# getResultsAndComputePAt10AtDifferentPruningLevels(numOfDocumentsIncluded)
# Updated by Wei on 20141101
inputFileName1 = sys.argv[1]
getResultsAndComputePAt10ForDocHits(inputFileName1)
print "Ends."






